# -*- coding:utf-8 -*-
# 整个项目的入口文件，主干
from html_downloader import HtmlDownloader
from url_manager import UrlManager
from html_outputer import HtmlOutputer
from html_parser import HtmlParser
import time      # 睡眠，防止反爬


class SpiderMain(object):
    def __init__(self):
        self.downloader = HtmlDownloader()
        self.urls = UrlManager()
        self.outputer = HtmlOutputer()
        self.parser = HtmlParser()

    def craw(self, root_url, page_amount=5, time_sleep=None):
        """爬取"""
        count = 1
        # 添加第一个待爬取url
        self.urls.add_new_url(root_url)
        # 如果集合中有url，那么就取出一个url请求，没有连接则跳出。
        while self.urls.has_new_url():
            # 开始爬取
            # 取一个待爬取的url
            try:
                new_url = self.urls.get_new_url()
                print(f'craw{count}:{new_url}')
                # 请求一个url，返回HTML
                html_content = self.downloader.download(new_url)
                # xpath解析HTML，得到需要的数据
                new_urls, new_data = self.parser.parse(html_content)
                # 一个词条页面上关联的a连接列表加入url管理器中待爬取
                self.urls.add_new_urls(new_urls)
                self.outputer.collect_data(new_url, new_data)

                count += 1
                if count > page_amount:
                    break

                time.sleep(time_sleep)

            except Exception as e:
                print(f'craw failed {new_url}')

        self.outputer.output_html()
        print('done')


if __name__ == '__main__':
    POOT_URL = 'https://baike.baidu.com/item/Python/407313'
    # 第一个要爬取的页面
    PAGE_AMOUNT = 2    # 总共请求多少页
    TIME_SLEEP = 2      # 睡眠时间过长比较慢，睡眠过短会封
    spider = SpiderMain()
    spider.craw(POOT_URL, PAGE_AMOUNT, TIME_SLEEP)




